from bs4 import BeautifulSoup
import requests
import pymongo

client = pymongo.MongoClient('localhost', 27017)
trip = client['trip']
bnb_info = trip['bnb_info']


def get_page_within(pages):
    for page_num in range(1,pages+1):
        url = 'https://www.tripadvisor.cn/Attractions-g60763-Activities-oa{}-New_York_City_New_York.html'.format((page_num-1)*30)
        wb_data = requests.get(url)
        soup = BeautifulSoup(wb_data.text, 'lxml')
        titles = soup.select('div.listing_title > a[target="_blank"]')
        imgs = soup.select('img[width="180"]')
        cates = soup.select('div.p13n_reasoning_v2')
        temps = soup.select('span.ui_bubble_rating')
        for title, img, cate, temp in zip(titles, imgs, cates, temps):
            cate_list = list(cate.stripped_strings)
            for one_cate in cate_list:
                if one_cate == ',':
                    cate_list.remove(one_cate)
            data = {
                'title': title.get_text(),
                'img': img.get('src'),
                'cate': cate_list,
                'temp': temp.get('alt')[0]
            }
            print(data)
            bnb_info.insert_one(data)
get_page_within(3) #获取前三页面得数据

